{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#  任务4 模型选择(3天)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 以lightGBM为例"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-12-24T13:53:49.093200Z",
     "start_time": "2019-12-24T13:53:43.498159Z"
    }
   },
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import lightgbm as lgb\n",
    "import sklearn\n",
    "import numpy\n",
    "import hyperopt\n",
    "from hyperopt import hp, fmin, tpe, STATUS_OK, Trials\n",
    "import colorama\n",
    "import numpy as np\n",
    "\n",
    "N_HYPEROPT_PROBES = 500\n",
    "HYPEROPT_ALGO = tpe.suggest  #  tpe.suggest OR hyperopt.rand.suggest\n",
    "\n",
    "# ----------------------------------------------------------\n",
    "\n",
    "colorama.init()\n",
    "\n",
    "# ---------------------------------------------------------------------\n",
    "\n",
    "def get_lgb_params(space):\n",
    "    lgb_params = dict()\n",
    "    lgb_params['boosting_type'] = space['boosting_type'] if 'boosting_type' in space else 'gbdt'\n",
    "    lgb_params['objective'] = 'regression'\n",
    "    lgb_params['metric'] = 'rmse'\n",
    "    lgb_params['learning_rate'] = space['learning_rate']\n",
    "    lgb_params['num_leaves'] = int(space['num_leaves'])\n",
    "    lgb_params['min_data_in_leaf'] = int(space['min_data_in_leaf'])\n",
    "    lgb_params['min_sum_hessian_in_leaf'] = space['min_sum_hessian_in_leaf']\n",
    "    lgb_params['max_depth'] = -1\n",
    "    lgb_params['lambda_l1'] = space['lambda_l1'] if 'lambda_l1' in space else 0.0\n",
    "    lgb_params['lambda_l2'] = space['lambda_l2'] if 'lambda_l2' in space else 0.0\n",
    "    lgb_params['max_bin'] = int(space['max_bin']) if 'max_bin' in space else 256\n",
    "    lgb_params['feature_fraction'] = space['feature_fraction']\n",
    "    lgb_params['bagging_fraction'] = space['bagging_fraction']\n",
    "    lgb_params['bagging_freq'] = int(space['bagging_freq']) if 'bagging_freq' in space else 1\n",
    "    lgb_params['nthread'] = 4\n",
    "    return lgb_params\n",
    "\n",
    "# ---------------------------------------------------------------------\n",
    "\n",
    "obj_call_count = 0\n",
    "cur_best_score = 0 # 0 or np.inf\n",
    "log_writer = open( '../log/lgb-hyperopt-log.txt', 'w' )\n",
    "\n",
    "\n",
    "def objective(space):\n",
    "    global obj_call_count, cur_best_score\n",
    "\n",
    "    obj_call_count += 1\n",
    "\n",
    "    print('\\nLightGBM objective call #{} cur_best_score={:7.5f}'.format(obj_call_count,cur_best_score) )\n",
    "\n",
    "    lgb_params = get_lgb_params(space)\n",
    "\n",
    "    sorted_params = sorted(space.items(), key=lambda z: z[0])\n",
    "    params_str = str.join(' ', ['{}={}'.format(k, v) for k, v in sorted_params])\n",
    "    print('Params: {}'.format(params_str) )\n",
    "    \n",
    "    kf = KFold(n_splits=3, shuffle=True, random_state=0)\n",
    "    out_of_fold = np.zeros(len(X_train))\n",
    "    for fold, (train_idx, val_idx) in enumerate(kf.split(X_train)):\n",
    "        D_train = lgb.Dataset(X_train.iloc[train_idx], label=Y_train[train_idx])\n",
    "        D_val = lgb.Dataset(X_train.iloc[val_idx], label=Y_train[val_idx])\n",
    "        # Train\n",
    "        num_round = 10000\n",
    "        clf = lgb.train(lgb_params,\n",
    "                           D_train,\n",
    "                           num_boost_round=num_round,\n",
    "                           # metrics='mlogloss',\n",
    "                           valid_sets=D_val,\n",
    "                           # valid_names='val',\n",
    "                           # fobj=None,\n",
    "                           # feval=None,\n",
    "                           # init_model=None,\n",
    "                           # feature_name='auto',\n",
    "                           # categorical_feature='auto',\n",
    "                           early_stopping_rounds=200,\n",
    "                           # evals_result=None,\n",
    "                           verbose_eval=False,\n",
    "                           # learning_rates=None,\n",
    "                           # keep_training_booster=False,\n",
    "                           # callbacks=None\n",
    "                           )\n",
    "        # predict\n",
    "        nb_trees = clf.best_iteration\n",
    "        val_loss = clf.best_score['valid_0']\n",
    "        print('nb_trees={} val_loss={}'.format(nb_trees, val_loss))\n",
    "        out_of_fold[val_idx] = clf.predict(X_train.iloc[val_idx], num_iteration=nb_trees)\n",
    "        score = r2_score(out_of_fold, Y_train)\n",
    "\n",
    "    print('val_r2_score={}'.format(score))\n",
    "\n",
    "    log_writer.write('score={} Params:{} nb_trees={}\\n'.format(score, params_str, nb_trees ))\n",
    "    log_writer.flush()\n",
    "\n",
    "    if score>cur_best_score:\n",
    "        cur_best_score = score\n",
    "        print(colorama.Fore.GREEN + 'NEW BEST SCORE={}'.format(cur_best_score) + colorama.Fore.RESET)\n",
    "    return {'loss': -score, 'status': STATUS_OK}\n",
    "\n",
    "# --------------------------------------------------------------------------------\n",
    "\n",
    "space ={\n",
    "        'num_leaves': hp.quniform ('num_leaves', 10, 100, 1),\n",
    "        'min_data_in_leaf':  hp.quniform ('min_data_in_leaf', 10, 100, 1),\n",
    "        'feature_fraction': hp.uniform('feature_fraction', 0.75, 1.0),\n",
    "        'bagging_fraction': hp.uniform('bagging_fraction', 0.75, 1.0),\n",
    "        'learning_rate': hp.uniform('learning_rate', 0, 0.01),\n",
    "#         'learning_rate': hp.loguniform('learning_rate', -5.0, -2.3),\n",
    "        'min_sum_hessian_in_leaf': hp.loguniform('min_sum_hessian_in_leaf', 0, 2.3),\n",
    "        'max_bin': hp.quniform ('max_bin', 88, 200, 1),\n",
    "        'bagging_freq': hp.quniform ('bagging_freq', 1, 15, 1),\n",
    "        'lambda_l1': hp.uniform('lambda_l1', 0, 10 ),\n",
    "        'lambda_l2': hp.uniform('lambda_l2', 0, 10 ),\n",
    "       }\n",
    "\n",
    "trials = Trials()\n",
    "best = hyperopt.fmin(fn=objective,\n",
    "                     space=space,\n",
    "                     algo=HYPEROPT_ALGO,\n",
    "                     max_evals=N_HYPEROPT_PROBES,\n",
    "                     trials=trials,\n",
    "                     verbose=1)\n",
    "\n",
    "print('-'*50)\n",
    "print('The best params:')\n",
    "print( best )\n",
    "print('\\n\\n')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
